package org.qengine.utils.Deduplicator;
import org.apache.commons.math3.linear.RealVector;
import org.qengine.entity.QuestionEntity;

import java.util.*;
/*
* 添加考题列表
* 记录考题的id和文本
* 返回考题的id列表
* */
public class QuestionDeduplicator {

    // 相似度阈值
    private static final double SIMILARITY_THRESHOLD = 0.4;
    //进行去重处理
    public static List<Integer> deduplicate(List<QuestionEntity> questionEntities) {
        List<String> processedQuestions = new ArrayList<>();
        List<Integer> result = new ArrayList<>();
        //初始化分词器
        TextPreprocessor preprocessor = new TextPreprocessor();
        Vectorizer vectorizer ;

        //提取题目id和题干文本
        List<Map<Integer,String>> questions = new ArrayList<>();
        List<String> questionTexts = new ArrayList<>();
        for (QuestionEntity questionEntity : questionEntities) {
            Map<Integer,String> question = new HashMap<>();
            String questionText = questionEntity.getQuestionStem();
            //文本预处理
            String processedQuestion = TextPreprocessor.preprocess(questionText);
            question.put(questionEntity.getId(),processedQuestion);
            questions.add(question);
            questionTexts.add(processedQuestion);
        }

        //初始化特征向量处理器
        vectorizer = new Vectorizer(questionTexts);

        for (Map<Integer,String> question : questions) {
            Integer id = question.keySet().iterator().next();
            String questionText = question.get(id);
            // 文本预处理
            // 计算 TF-IDF 向量
            RealVector firstVector = vectorizer.transform(questionText);
            // 判断是否重复
            for (Map<Integer,String> secondQuestion : questions) {
                Integer secondId = secondQuestion.keySet().iterator().next();
                String secondQuestionText = secondQuestion.get(secondId);
                if (Objects.equals(id, secondId)) {
                    continue;
                }
                RealVector secondVector = vectorizer.transform(secondQuestionText);
                double similarity = SimilarityCalculator.cosineSimilarity(firstVector, secondVector);
                // 如果相似度大于阈值，则认为重复,将id大的一方加入结果列表
                if (similarity > SIMILARITY_THRESHOLD) {
                    if (id > secondId) {
                        if(!result.contains(id)) {
                            result.add(id);
                        }
                    }
                    else {
                        if(!result.contains(secondId)) {
                            result.add(secondId);
                        }
                    }
                    break;
                }
            }
        }
        return result;
    }
}
